# -*- coding: utf-8 -*-
# -文件说明  通用框架-
# @Time    : 2019/8/8 16:28
# @Author  : hwx
# @Email   : @163.com
# @File    : 爬虫demoV1.py
# @Software: PyCharm

import requests

# 无参数的url


def getHTMLText(url):
    try:
        r = requests.get(url, timeout=10)
        erro = r.raise_for_status()  # 如果不是200，抛出异常
        r.encoding = r.apparent_encoding
        return r.text
    except BaseException:
        # print(erro.)
        return "getHTMLText产生异常"
    # 有参数的url


def getHTMLText(url, paramdate):
    try:
        r = requests.get(url, data=paramdate)
        headmsg = r.request.headers
        print(headmsg)
        print('状态码：' + r.status_code)
        r.raise_for_status()  # 如果不是200，抛出异常
        r.encoding = r.apparent_encoding
        return r.text
    except BaseException:
        return "getHTMLText产生异常"

    # 请求头


def getHTMLText3(url, herdersmsg):
    try:
        '''
                因为淘宝反爬虫机制,必须有cookie，和user-agent
         '''
        headersmsg = {
            'user-agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36',
            'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3',
            'cookie': 'thw=cn; cna=T5QxFT75MFwCAd9SzD64M3Tw; tracknick=%5Cu517C%5Cu804C%5Cu7684%5Cu5C0F%5Cu9EC4; _cc_=UIHiLt3xSw%3D%3D; tg=0; enc=dtrgSGswkvO%2FsWQ%2FZj4vxeXZyW74wg%2FFFHIKbCvJMyKyl5N%2B7Iv6P046Z8TAYAhnYHwQ3I%2FaxNlU52fyWCq8FQ%3D%3D; hng=CN%7Czh-CN%7CCNY%7C156; x=e%3D1%26p%3D*%26s%3D0%26c%3D0%26f%3D0%26g%3D0%26t%3D0; miid=374127122081873098; v=0; t=643574ecf918da2f3deafbfb77874196; cookie2=11997510615e192d0ce4bfe81d228ba2; _tb_token_=e85e170df53d7; alitrackid=www.taobao.com; lastalitrackid=www.taobao.com; JSESSIONID=052B40B78F375AE49686595F78ED6957; l=cBrYfBbrvnV-ct4BBOCgquI8mf7tIIRAguPRwEivi_5Qa6L6pMbOkDwuMFp6cjWdteLB4keiqTy9-etkiKy06Pt-g3fP.; isg=BKys-oQV3VJ4NMjoRirOQNIlfYoezTR4Qz5wVgbtvtf6EUwbLnFnn_GjMZkMmYhn'}
        r = requests.get(url, headers=headersmsg, timeout=30)  # 请用get，
        print('状态码：' + str(r.status_code))
        r.raise_for_status()  # 如果不是200，抛出异常
        r.encoding = r.apparent_encoding
        return r.text
    except BaseException:
        return "getHTMLText产生异常"


url = "http://www.qq.com/robots.txt"  # www.baidu.com
print(getHTMLText(url))
